{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import category_encoders as ce\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "import category_encoders as ce\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn import preprocessing\n",
    "from keras.layers import Input, Dropout, Dense, Embedding, SpatialDropout1D, concatenate\n",
    "import keras as ks\n",
    "from keras import backend as K\n",
    "import gc\n",
    "from contextlib import contextmanager\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "num_rows = None\n",
    "EPS = 1e-100\n",
    "\n",
    "\n",
    "@contextmanager\n",
    "def timer(name):\n",
    "    t0 = time.time()\n",
    "    yield\n",
    "    print('[{' + name + '}] done in {' + str(round(time.time() - t0, 3)) + '} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hot encoding for categorical columns with get_dummies\n",
    "def one_hot_encoder(df, nan_as_category = True):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df, new_columns\n",
    "\n",
    "# Preprocess application_train.csv and application_test.csv\n",
    "def application_train_test(num_rows = None, nan_as_category = False):\n",
    "    # Read data and merge\n",
    "    df = pd.read_csv('../data/application_train.csv', nrows= num_rows)\n",
    "    test_df = pd.read_csv('../data/application_test.csv', nrows= num_rows)\n",
    "    print(\"Train samples: {}, test samples: {}\".format(len(df), len(test_df)))\n",
    "    df = df.append(test_df).reset_index()\n",
    "    # Optional: Remove 4 applications with XNA CODE_GENDER (train set)\n",
    "   \n",
    "    \n",
    "    docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]\n",
    "    live = [_f for _f in df.columns if ('FLAG_' in _f) & ('FLAG_DOC' not in _f) & ('_FLAG_' not in _f)]\n",
    "    \n",
    "    # NaN values for DAYS_EMPLOYED: 365.243 -> nan\n",
    "    df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)\n",
    "\n",
    "    inc_by_org = df[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE').median()['AMT_INCOME_TOTAL']\n",
    "\n",
    "    df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']\n",
    "    df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']\n",
    "    df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)\n",
    "    df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)\n",
    "    df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] / (1 + df['CNT_CHILDREN'])\n",
    "    df['NEW_INC_BY_ORG'] = df['ORGANIZATION_TYPE'].map(inc_by_org)\n",
    "    df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']\n",
    "    df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] / (1 + df['AMT_INCOME_TOTAL'])\n",
    "    df['NEW_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']\n",
    "    df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)\n",
    "    df['NEW_SCORES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)\n",
    "    df['NEW_SCORES_STD'] = df['NEW_SCORES_STD'].fillna(df['NEW_SCORES_STD'].mean())\n",
    "    df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']\n",
    "    df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']\n",
    "    df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']\n",
    "    df['NEW_PHONE_TO_BIRTH_RATIO_EMPLOYER'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']\n",
    "    df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']\n",
    "    \n",
    "    # Categorical features with Binary encode (0 or 1; two categories)\n",
    "    for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:\n",
    "        df[bin_feature], uniques = pd.factorize(df[bin_feature])\n",
    "    # Categorical features with One-Hot encode\n",
    "    df, cat_cols = one_hot_encoder(df, nan_as_category)\n",
    "    dropcolum=['FLAG_DOCUMENT_2','FLAG_DOCUMENT_4',\n",
    "    'FLAG_DOCUMENT_5','FLAG_DOCUMENT_6','FLAG_DOCUMENT_7',\n",
    "    'FLAG_DOCUMENT_8','FLAG_DOCUMENT_9','FLAG_DOCUMENT_10', \n",
    "    'FLAG_DOCUMENT_11','FLAG_DOCUMENT_12','FLAG_DOCUMENT_13',\n",
    "    'FLAG_DOCUMENT_14','FLAG_DOCUMENT_15','FLAG_DOCUMENT_16',\n",
    "    'FLAG_DOCUMENT_17','FLAG_DOCUMENT_18','FLAG_DOCUMENT_19',\n",
    "    'FLAG_DOCUMENT_20','FLAG_DOCUMENT_21']\n",
    "    df= df.drop(dropcolum,axis=1)\n",
    "    del test_df\n",
    "    gc.collect()\n",
    "    return df\n",
    "\n",
    "# Preprocess bureau.csv and bureau_balance.csv\n",
    "def bureau_and_balance(num_rows = None, nan_as_category = True):\n",
    "    bureau = pd.read_csv('../data/bureau.csv', nrows = num_rows)\n",
    "    bb = pd.read_csv('../data/bureau_balance.csv', nrows = num_rows)\n",
    "    bb, bb_cat = one_hot_encoder(bb, nan_as_category)\n",
    "    bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)\n",
    "    \n",
    "    # Bureau balance: Perform aggregations and merge with bureau.csv\n",
    "    bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}\n",
    "    for col in bb_cat:\n",
    "        bb_aggregations[col] = ['mean']\n",
    "    bb_agg = bb.groupby('SK_ID_BUREAU').agg(bb_aggregations)\n",
    "    bb_agg.columns = pd.Index([e[0] + \"_\" + e[1].upper() for e in bb_agg.columns.tolist()])\n",
    "    bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')\n",
    "    bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)\n",
    "    del bb, bb_agg\n",
    "    gc.collect()\n",
    "    \n",
    "    # Bureau and bureau_balance numeric features\n",
    "    num_aggregations = {\n",
    "        'DAYS_CREDIT': [ 'mean', 'var'],\n",
    "        'DAYS_CREDIT_ENDDATE': [ 'mean'],\n",
    "        'DAYS_CREDIT_UPDATE': ['mean'],\n",
    "        'CREDIT_DAY_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_MAX_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_SUM': [ 'mean', 'sum'],\n",
    "        'AMT_CREDIT_SUM_DEBT': [ 'mean', 'sum'],\n",
    "        'AMT_CREDIT_SUM_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],\n",
    "        'AMT_ANNUITY': ['max', 'mean'],\n",
    "        'CNT_CREDIT_PROLONG': ['sum'],\n",
    "        'MONTHS_BALANCE_MIN': ['min'],\n",
    "        'MONTHS_BALANCE_MAX': ['max'],\n",
    "        'MONTHS_BALANCE_SIZE': ['mean', 'sum']\n",
    "    }\n",
    "    # Bureau and bureau_balance categorical features\n",
    "    cat_aggregations = {}\n",
    "    for cat in bureau_cat: cat_aggregations[cat] = ['mean']\n",
    "    for cat in bb_cat: cat_aggregations[cat + \"_MEAN\"] = ['mean']\n",
    "    \n",
    "    bureau_agg = bureau.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})\n",
    "    bureau_agg.columns = pd.Index(['BURO_' + e[0] + \"_\" + e[1].upper() for e in bureau_agg.columns.tolist()])\n",
    "    # Bureau: Active credits - using only numerical aggregations\n",
    "    active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]\n",
    "    active_agg = active.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    active_agg.columns = pd.Index(['ACTIVE_' + e[0] + \"_\" + e[1].upper() for e in active_agg.columns.tolist()])\n",
    "    bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')\n",
    "    del active, active_agg\n",
    "    gc.collect()\n",
    "    # Bureau: Closed credits - using only numerical aggregations\n",
    "    closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]\n",
    "    closed_agg = closed.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    closed_agg.columns = pd.Index(['CLOSED_' + e[0] + \"_\" + e[1].upper() for e in closed_agg.columns.tolist()])\n",
    "    bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')\n",
    "    del closed, closed_agg, bureau\n",
    "    gc.collect()\n",
    "    return bureau_agg\n",
    "\n",
    "# Preprocess previous_applications.csv\n",
    "def previous_applications(num_rows = None, nan_as_category = True):\n",
    "    prev = pd.read_csv('../data/previous_application.csv', nrows = num_rows)\n",
    "    prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)\n",
    "    # Days 365.243 values -> nan\n",
    "    prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)\n",
    "    # Add feature: value ask / value received percentage\n",
    "    prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']\n",
    "    # Previous applications numeric features\n",
    "    num_aggregations = {\n",
    "        'AMT_ANNUITY': [ 'max', 'mean'],\n",
    "        'AMT_APPLICATION': [ 'max','mean'],\n",
    "        'AMT_CREDIT': [ 'max', 'mean'],\n",
    "        'APP_CREDIT_PERC': [ 'max', 'mean'],\n",
    "        'AMT_DOWN_PAYMENT': [ 'max', 'mean'],\n",
    "        'AMT_GOODS_PRICE': [ 'max', 'mean'],\n",
    "        'HOUR_APPR_PROCESS_START': [ 'max', 'mean'],\n",
    "        'RATE_DOWN_PAYMENT': [ 'max', 'mean'],\n",
    "        'DAYS_DECISION': [ 'max', 'mean'],\n",
    "        'CNT_PAYMENT': ['mean', 'sum'],\n",
    "    }\n",
    "    # Previous applications categorical features\n",
    "    cat_aggregations = {}\n",
    "    for cat in cat_cols:\n",
    "        cat_aggregations[cat] = ['mean']\n",
    "    \n",
    "    prev_agg = prev.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})\n",
    "    prev_agg.columns = pd.Index(['PREV_' + e[0] + \"_\" + e[1].upper() for e in prev_agg.columns.tolist()])\n",
    "    # Previous Applications: Approved Applications - only numerical features\n",
    "    approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]\n",
    "    approved_agg = approved.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    approved_agg.columns = pd.Index(['APPROVED_' + e[0] + \"_\" + e[1].upper() for e in approved_agg.columns.tolist()])\n",
    "    prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')\n",
    "    # Previous Applications: Refused Applications - only numerical features\n",
    "    refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]\n",
    "    refused_agg = refused.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    refused_agg.columns = pd.Index(['REFUSED_' + e[0] + \"_\" + e[1].upper() for e in refused_agg.columns.tolist()])\n",
    "    prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')\n",
    "    del refused, refused_agg, approved, approved_agg, prev\n",
    "    gc.collect()\n",
    "    return prev_agg\n",
    "\n",
    "# Preprocess POS_CASH_balance.csv\n",
    "def pos_cash(num_rows = None, nan_as_category = True):\n",
    "    pos = pd.read_csv('../data/POS_CASH_balance.csv', nrows = num_rows)\n",
    "    pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)\n",
    "    # Features\n",
    "    aggregations = {\n",
    "        'MONTHS_BALANCE': ['max', 'mean', 'size'],\n",
    "        'SK_DPD': ['max', 'mean'],\n",
    "        'SK_DPD_DEF': ['max', 'mean']\n",
    "    }\n",
    "    for cat in cat_cols:\n",
    "        aggregations[cat] = ['mean']\n",
    "    \n",
    "    pos_agg = pos.groupby('SK_ID_CURR').agg(aggregations)\n",
    "    pos_agg.columns = pd.Index(['POS_' + e[0] + \"_\" + e[1].upper() for e in pos_agg.columns.tolist()])\n",
    "    # Count pos cash accounts\n",
    "    pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR').size()\n",
    "    del pos\n",
    "    gc.collect()\n",
    "    return pos_agg\n",
    "    \n",
    "# Preprocess installments_payments.csv\n",
    "def installments_payments(num_rows = None, nan_as_category = True):\n",
    "    ins = pd.read_csv('../data/installments_payments.csv', nrows = num_rows)\n",
    "    ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)\n",
    "    # Percentage and difference paid in each installment (amount paid and installment value)\n",
    "    ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']\n",
    "    ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']\n",
    "    # Days past due and days before due (no negative values)\n",
    "    ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']\n",
    "    ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']\n",
    "    ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)\n",
    "    ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)\n",
    "    # Features: Perform aggregations\n",
    "    aggregations = {\n",
    "        'NUM_INSTALMENT_VERSION': ['nunique'],\n",
    "        'DPD': ['max', 'mean', 'sum'],\n",
    "        'DBD': ['max', 'mean', 'sum'],\n",
    "        'PAYMENT_PERC': [ 'mean',  'var'],\n",
    "        'PAYMENT_DIFF': [ 'mean', 'var'],\n",
    "        'AMT_INSTALMENT': ['max', 'mean', 'sum'],\n",
    "        'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],\n",
    "        'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']\n",
    "    }\n",
    "    for cat in cat_cols:\n",
    "        aggregations[cat] = ['mean']\n",
    "    ins_agg = ins.groupby('SK_ID_CURR').agg(aggregations)\n",
    "    ins_agg.columns = pd.Index(['INSTAL_' + e[0] + \"_\" + e[1].upper() for e in ins_agg.columns.tolist()])\n",
    "    # Count installments accounts\n",
    "    ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR').size()\n",
    "    del ins\n",
    "    gc.collect()\n",
    "    return ins_agg\n",
    "\n",
    "# Preprocess credit_card_balance.csv\n",
    "def credit_card_balance(num_rows = None, nan_as_category = True):\n",
    "    cc = pd.read_csv('../data/credit_card_balance.csv', nrows = num_rows)\n",
    "    cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)\n",
    "    # General aggregations\n",
    "    cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)\n",
    "    cc_agg = cc.groupby('SK_ID_CURR').agg([ 'max', 'mean', 'sum', 'var'])\n",
    "    cc_agg.columns = pd.Index(['CC_' + e[0] + \"_\" + e[1].upper() for e in cc_agg.columns.tolist()])\n",
    "    # Count credit card lines\n",
    "    cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR').size()\n",
    "    del cc\n",
    "    gc.collect()\n",
    "    return cc_agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples: 307511, test samples: 48744\n"
     ]
    }
   ],
   "source": [
    "df = application_train_test(num_rows=num_rows, nan_as_category=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 240)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau = bureau_and_balance(num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(bureau, how='left', on='SK_ID_CURR')\n",
    "del bureau\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Previous applications df shape: (338857, 219)\n",
      "Pos-cash balance df shape: (337252, 18)\n",
      "Installments payments df shape: (339587, 22)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev = previous_applications(num_rows)\n",
    "print(\"Previous applications df shape:\", prev.shape)\n",
    "df = df.join(prev, how='left', on='SK_ID_CURR')\n",
    "del prev\n",
    "gc.collect()\n",
    "\n",
    "pos = pos_cash(num_rows)\n",
    "print(\"Pos-cash balance df shape:\", pos.shape)\n",
    "df = df.join(pos, how='left', on='SK_ID_CURR')\n",
    "del pos\n",
    "gc.collect()\n",
    "\n",
    "ins = installments_payments(num_rows)\n",
    "print(\"Installments payments df shape:\", ins.shape)\n",
    "df = df.join(ins, how='left', on='SK_ID_CURR')\n",
    "del ins\n",
    "gc.collect()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Credit card balance df shape: (103558, 113)\n",
      "(356255, 707)\n"
     ]
    }
   ],
   "source": [
    "cc = credit_card_balance(num_rows)\n",
    "print(\"Credit card balance df shape:\", cc.shape)\n",
    "df = df.join(cc, how='left', on='SK_ID_CURR')\n",
    "del cc\n",
    "gc.collect()\n",
    "\n",
    "print(df.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 707)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack/test_ann_1.csv\"\n",
    "validation_file_path = 'Level_1_stack/validation_ann_1.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip', nrows= num_rows)\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 707)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def safe_int(x):\n",
    "    try:\n",
    "        res = int(x)\n",
    "    except:\n",
    "        res = 0\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from keras.callbacks import Callback\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "class RocAucEvaluation(Callback):\n",
    "    def __init__(self, validation_data=(), interval=1):\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "        self.interval = interval\n",
    "        self.X_val, self.y_val = validation_data\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if epoch % self.interval == 0:\n",
    "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "            score = roc_auc_score(self.y_val, y_pred)\n",
    "            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting LightGBM. Train shape: (307511, 707), test shape: (48744, 707)\n",
      "(307511, 704)\n",
      "(246008, 704) (61503, 704) (48744, 704)\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/20\n",
      "246008/246008 [==============================] - 87s 353us/step - loss: 0.2791 - binary_crossentropy: 0.2791 - val_loss: 0.2578 - val_binary_crossentropy: 0.2578\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.736101\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25783, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2553 - binary_crossentropy: 0.2553 - val_loss: 0.2540 - val_binary_crossentropy: 0.2540\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.750660\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25783 to 0.25399, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2526 - binary_crossentropy: 0.2526 - val_loss: 0.2550 - val_binary_crossentropy: 0.2550\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.751125\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.25399\n",
      "Epoch 4/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2513 - binary_crossentropy: 0.2513 - val_loss: 0.2520 - val_binary_crossentropy: 0.2520\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.755971\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.25399 to 0.25203, saving model to best_model.hdf5\n",
      "Epoch 5/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2506 - binary_crossentropy: 0.2506 - val_loss: 0.2538 - val_binary_crossentropy: 0.2538\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.752806\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.25203\n",
      "Epoch 6/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2492 - binary_crossentropy: 0.2492 - val_loss: 0.2515 - val_binary_crossentropy: 0.2515\n",
      "\n",
      " ROC-AUC - epoch: 6 - score: 0.758404\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.25203 to 0.25153, saving model to best_model.hdf5\n",
      "Epoch 7/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2492 - binary_crossentropy: 0.2492 - val_loss: 0.2506 - val_binary_crossentropy: 0.2506\n",
      "\n",
      " ROC-AUC - epoch: 7 - score: 0.758827\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.25153 to 0.25063, saving model to best_model.hdf5\n",
      "Epoch 8/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2486 - binary_crossentropy: 0.2486 - val_loss: 0.2503 - val_binary_crossentropy: 0.2503\n",
      "\n",
      " ROC-AUC - epoch: 8 - score: 0.760264\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.25063 to 0.25030, saving model to best_model.hdf5\n",
      "Epoch 9/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2480 - binary_crossentropy: 0.2480 - val_loss: 0.2500 - val_binary_crossentropy: 0.2500\n",
      "\n",
      " ROC-AUC - epoch: 9 - score: 0.760102\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.25030 to 0.24998, saving model to best_model.hdf5\n",
      "Epoch 10/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2471 - binary_crossentropy: 0.2471 - val_loss: 0.2512 - val_binary_crossentropy: 0.2512\n",
      "\n",
      " ROC-AUC - epoch: 10 - score: 0.760892\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.24998\n",
      "Epoch 11/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2469 - binary_crossentropy: 0.2469 - val_loss: 0.2509 - val_binary_crossentropy: 0.2509\n",
      "\n",
      " ROC-AUC - epoch: 11 - score: 0.761451\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.24998\n",
      "Epoch 12/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2469 - binary_crossentropy: 0.2469 - val_loss: 0.2497 - val_binary_crossentropy: 0.2497\n",
      "\n",
      " ROC-AUC - epoch: 12 - score: 0.761166\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.24998 to 0.24975, saving model to best_model.hdf5\n",
      "Epoch 13/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2463 - binary_crossentropy: 0.2463 - val_loss: 0.2507 - val_binary_crossentropy: 0.2507\n",
      "\n",
      " ROC-AUC - epoch: 13 - score: 0.762448\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.24975\n",
      "Epoch 14/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2459 - binary_crossentropy: 0.2459 - val_loss: 0.2508 - val_binary_crossentropy: 0.2508\n",
      "\n",
      " ROC-AUC - epoch: 14 - score: 0.762489\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.24975\n",
      "Epoch 15/20\n",
      "246008/246008 [==============================] - 86s 350us/step - loss: 0.2452 - binary_crossentropy: 0.2452 - val_loss: 0.2492 - val_binary_crossentropy: 0.2492\n",
      "\n",
      " ROC-AUC - epoch: 15 - score: 0.763063\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.24975 to 0.24923, saving model to best_model.hdf5\n",
      "Epoch 16/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2451 - binary_crossentropy: 0.2451 - val_loss: 0.2494 - val_binary_crossentropy: 0.2494\n",
      "\n",
      " ROC-AUC - epoch: 16 - score: 0.761349\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.24923\n",
      "Epoch 17/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2451 - binary_crossentropy: 0.2451 - val_loss: 0.2496 - val_binary_crossentropy: 0.2496\n",
      "\n",
      " ROC-AUC - epoch: 17 - score: 0.761533\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.24923\n",
      "Epoch 18/20\n",
      "246008/246008 [==============================] - 86s 351us/step - loss: 0.2447 - binary_crossentropy: 0.2447 - val_loss: 0.2498 - val_binary_crossentropy: 0.2498\n",
      "\n",
      " ROC-AUC - epoch: 18 - score: 0.763085\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.24923\n",
      "[{pass 1}] done in {1795.16} s\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/20\n",
      "246008/246008 [==============================] - 80s 324us/step - loss: 0.2431 - binary_crossentropy: 0.2431 - val_loss: 0.2490 - val_binary_crossentropy: 0.2490\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.765052\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24923 to 0.24895, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246008/246008 [==============================] - 80s 324us/step - loss: 0.2422 - binary_crossentropy: 0.2422 - val_loss: 0.2484 - val_binary_crossentropy: 0.2484\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.765140\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24895 to 0.24836, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246008/246008 [==============================] - 80s 324us/step - loss: 0.2420 - binary_crossentropy: 0.2420 - val_loss: 0.2484 - val_binary_crossentropy: 0.2484\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.765411\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24836\n",
      "Epoch 4/20\n",
      "246008/246008 [==============================] - 80s 323us/step - loss: 0.2417 - binary_crossentropy: 0.2417 - val_loss: 0.2486 - val_binary_crossentropy: 0.2486\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.764851\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24836\n",
      "Epoch 5/20\n",
      "246008/246008 [==============================] - 80s 324us/step - loss: 0.2416 - binary_crossentropy: 0.2416 - val_loss: 0.2488 - val_binary_crossentropy: 0.2488\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.765224\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.24836\n",
      "[{pass 2}] done in {464.91} s\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2409 - binary_crossentropy: 0.2409 - val_loss: 0.2488 - val_binary_crossentropy: 0.2488\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.765722\n",
      "\n",
      "Epoch 00001: val_loss did not improve from 0.24836\n",
      "Epoch 2/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2411 - binary_crossentropy: 0.2411 - val_loss: 0.2484 - val_binary_crossentropy: 0.2484\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.765563\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 0.24836\n",
      "Epoch 3/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2408 - binary_crossentropy: 0.2408 - val_loss: 0.2486 - val_binary_crossentropy: 0.2486\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.765418\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24836\n",
      "Epoch 4/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2402 - binary_crossentropy: 0.2402 - val_loss: 0.2487 - val_binary_crossentropy: 0.2487\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.765177\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24836\n",
      "Epoch 5/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2405 - binary_crossentropy: 0.2405 - val_loss: 0.2483 - val_binary_crossentropy: 0.2483\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.765311\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24836 to 0.24832, saving model to best_model.hdf5\n",
      "Epoch 6/20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2401 - binary_crossentropy: 0.2401 - val_loss: 0.2484 - val_binary_crossentropy: 0.2484\n",
      "\n",
      " ROC-AUC - epoch: 6 - score: 0.765639\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.24832\n",
      "Epoch 7/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2402 - binary_crossentropy: 0.2402 - val_loss: 0.2503 - val_binary_crossentropy: 0.2503\n",
      "\n",
      " ROC-AUC - epoch: 7 - score: 0.763430\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.24832\n",
      "Epoch 8/20\n",
      "246008/246008 [==============================] - 79s 320us/step - loss: 0.2398 - binary_crossentropy: 0.2398 - val_loss: 0.2485 - val_binary_crossentropy: 0.2485\n",
      "\n",
      " ROC-AUC - epoch: 8 - score: 0.765312\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.24832\n",
      "[{pass 3}] done in {736.632} s\n",
      "0.7653120255261938\n",
      "[{fit_predict}] done in {3034.161} s\n",
      "(246009, 704) (61502, 704) (48744, 704)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/20\n",
      "246009/246009 [==============================] - 86s 350us/step - loss: 0.2798 - binary_crossentropy: 0.2798 - val_loss: 0.2537 - val_binary_crossentropy: 0.2537\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.738772\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25373, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2568 - binary_crossentropy: 0.2568 - val_loss: 0.2511 - val_binary_crossentropy: 0.2511\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.748535\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25373 to 0.25115, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2539 - binary_crossentropy: 0.2539 - val_loss: 0.2505 - val_binary_crossentropy: 0.2505\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.751895\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.25115 to 0.25052, saving model to best_model.hdf5\n",
      "Epoch 4/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2520 - binary_crossentropy: 0.2520 - val_loss: 0.2504 - val_binary_crossentropy: 0.2504\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.748741\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.25052 to 0.25036, saving model to best_model.hdf5\n",
      "Epoch 5/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2512 - binary_crossentropy: 0.2512 - val_loss: 0.2500 - val_binary_crossentropy: 0.2500\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.756987\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.25036 to 0.24997, saving model to best_model.hdf5\n",
      "Epoch 6/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2504 - binary_crossentropy: 0.2504 - val_loss: 0.2489 - val_binary_crossentropy: 0.2489\n",
      "\n",
      " ROC-AUC - epoch: 6 - score: 0.756449\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.24997 to 0.24892, saving model to best_model.hdf5\n",
      "Epoch 7/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2498 - binary_crossentropy: 0.2498 - val_loss: 0.2480 - val_binary_crossentropy: 0.2480\n",
      "\n",
      " ROC-AUC - epoch: 7 - score: 0.757496\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.24892 to 0.24798, saving model to best_model.hdf5\n",
      "Epoch 8/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2495 - binary_crossentropy: 0.2495 - val_loss: 0.2481 - val_binary_crossentropy: 0.2481\n",
      "\n",
      " ROC-AUC - epoch: 8 - score: 0.758292\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.24798\n",
      "Epoch 9/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2488 - binary_crossentropy: 0.2488 - val_loss: 0.2476 - val_binary_crossentropy: 0.2476\n",
      "\n",
      " ROC-AUC - epoch: 9 - score: 0.759343\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.24798 to 0.24760, saving model to best_model.hdf5\n",
      "Epoch 10/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2479 - binary_crossentropy: 0.2479 - val_loss: 0.2470 - val_binary_crossentropy: 0.2470\n",
      "\n",
      " ROC-AUC - epoch: 10 - score: 0.759428\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.24760 to 0.24700, saving model to best_model.hdf5\n",
      "Epoch 11/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2476 - binary_crossentropy: 0.2476 - val_loss: 0.2471 - val_binary_crossentropy: 0.2471\n",
      "\n",
      " ROC-AUC - epoch: 11 - score: 0.760207\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.24700\n",
      "Epoch 12/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2475 - binary_crossentropy: 0.2475 - val_loss: 0.2470 - val_binary_crossentropy: 0.2470\n",
      "\n",
      " ROC-AUC - epoch: 12 - score: 0.760056\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.24700 to 0.24700, saving model to best_model.hdf5\n",
      "Epoch 13/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2468 - binary_crossentropy: 0.2468 - val_loss: 0.2466 - val_binary_crossentropy: 0.2466\n",
      "\n",
      " ROC-AUC - epoch: 13 - score: 0.761486\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.24700 to 0.24665, saving model to best_model.hdf5\n",
      "Epoch 14/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2467 - binary_crossentropy: 0.2467 - val_loss: 0.2465 - val_binary_crossentropy: 0.2465\n",
      "\n",
      " ROC-AUC - epoch: 14 - score: 0.761680\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.24665 to 0.24647, saving model to best_model.hdf5\n",
      "Epoch 15/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2461 - binary_crossentropy: 0.2461 - val_loss: 0.2465 - val_binary_crossentropy: 0.2465\n",
      "\n",
      " ROC-AUC - epoch: 15 - score: 0.761002\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.24647\n",
      "Epoch 16/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2459 - binary_crossentropy: 0.2459 - val_loss: 0.2479 - val_binary_crossentropy: 0.2479\n",
      "\n",
      " ROC-AUC - epoch: 16 - score: 0.763211\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.24647\n",
      "Epoch 17/20\n",
      "246009/246009 [==============================] - 86s 349us/step - loss: 0.2457 - binary_crossentropy: 0.2457 - val_loss: 0.2473 - val_binary_crossentropy: 0.2473\n",
      "\n",
      " ROC-AUC - epoch: 17 - score: 0.762856\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.24647\n",
      "[{pass 1}] done in {1693.61} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/20\n",
      "246009/246009 [==============================] - 79s 322us/step - loss: 0.2442 - binary_crossentropy: 0.2442 - val_loss: 0.2455 - val_binary_crossentropy: 0.2455\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.765025\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24647 to 0.24548, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246009/246009 [==============================] - 79s 322us/step - loss: 0.2434 - binary_crossentropy: 0.2434 - val_loss: 0.2455 - val_binary_crossentropy: 0.2455\n"
     ]
    }
   ],
   "source": [
    "encoding = 'ohe'\n",
    "\n",
    "train_df = df.iloc[0:n_train]\n",
    "test_df = df.iloc[n_train:]\n",
    "\n",
    "print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "feature_importance_df = pd.DataFrame()\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "#feats = [col for col in feats_0 if df[col].dtype == 'object']\n",
    "\n",
    "\n",
    "print(train_df[feats].shape)\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):\n",
    "        \n",
    "      \n",
    "        categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "        \n",
    "        if encoding == 'ohe':\n",
    "            \n",
    "            enc = ce.OneHotEncoder(impute_missing=True, cols=categorical_columns).fit(train_df[feats].iloc[train_idx],\n",
    "                                                                                       train_df['TARGET'].iloc[train_idx])\n",
    "            x_train = enc.transform(train_df[feats].iloc[train_idx]).replace([-np.inf, np.inf]).fillna(-999)\n",
    "            x_valid = enc.transform(train_df[feats].iloc[valid_idx]).replace([-np.inf, np.inf]).fillna(-999)\n",
    "            x_test = enc.transform(test_df[feats]).replace([-np.inf, np.inf]).fillna(-999)\n",
    "            gc.collect()\n",
    "            scaler = preprocessing.RobustScaler(quantile_range=(5.0, 95.0), with_scaling=True, with_centering=True)\n",
    "            scaler.fit(x_train)\n",
    "            x_train = scaler.transform(x_train)\n",
    "            x_valid = scaler.transform(x_valid)\n",
    "            x_test = scaler.transform(x_test)\n",
    "            \n",
    "            print(x_train.shape, x_valid.shape, x_test.shape)\n",
    "        \n",
    "        file_path = \"best_model.hdf5\"\n",
    "        check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose=1,\n",
    "                              save_best_only=True, mode=\"min\")\n",
    "        ra_val = RocAucEvaluation(validation_data=(x_valid, train_df['TARGET'].iloc[valid_idx].values), interval=1)\n",
    "        early_stop = EarlyStopping(monitor=\"val_loss\", mode = \"min\", patience=3)\n",
    "        gc.collect()\n",
    "        \n",
    "        config = tf.ConfigProto(\n",
    "        intra_op_parallelism_threads=6, use_per_session_threads=6, inter_op_parallelism_threads=6)\n",
    "        with tf.Session(graph=tf.Graph(), config=config) as sess, timer('fit_predict'):\n",
    "                    ks.backend.set_session(sess)\n",
    "                    model_in = ks.Input(shape=(x_train.shape[1],), dtype='float32', sparse=False)\n",
    "                    out = ks.layers.Dense(2 ** 12,  activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(model_in)\n",
    "                    out = ks.layers.Dropout(0.6)(out)\n",
    "                    out =  ks.layers.Dense(2 ** 9, activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    out = ks.layers.Dropout(0.3)(out)\n",
    "                    out =  ks.layers.Dense(2 ** 8, activation='relu', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    out = ks.layers.Dropout(0.3)(out)\n",
    "                    out = ks.layers.Dense(1, activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    model = ks.models.Model(model_in, out)\n",
    "                    model.compile(loss='binary_crossentropy',\n",
    "                                  optimizer=ks.optimizers.Adam(lr=1e-3), metrics=['binary_crossentropy'])\n",
    "                    batch_size = 2 ** 10\n",
    "                    for i in range(3):\n",
    "                        with timer('pass ' +  str(i + 1)):\n",
    "                            model.fit(x=x_train, y=train_df['TARGET'].iloc[train_idx].values, batch_size=batch_size+(batch_size*(2*i)), epochs=20, \n",
    "                                validation_data=(x_valid, train_df['TARGET'].iloc[valid_idx].values), callbacks=[ra_val, check_point, early_stop], shuffle=True)\n",
    "                    y_pred = model.predict(x_valid).reshape(-1, 1)\n",
    "                    print(roc_auc_score(y_true=train_df['TARGET'].iloc[valid_idx].values, y_score=y_pred))\n",
    "                    gc.collect()   \n",
    "                    oof_preds[valid_idx] = model.predict(x_valid)[:, 0]\n",
    "                    sub_preds += model.predict(x_test)[:, 0] / folds.n_splits\n",
    "                    gc.collect()\n",
    "\n",
    "print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))\n",
    "             \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Full AUC score 0.763312\n"
     ]
    }
   ],
   "source": [
    "print('Full AUC score %.6f' % roc_auc_score(list(map(safe_int, train_df['TARGET'])), oof_preds))\n",
    "        \n",
    "sub_df = test_df[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "val_df = train_df[['SK_ID_CURR', 'TARGET']].copy()\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)        \n",
    "            \n",
    "            \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48740,)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511,)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oof_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
